/*
 * This class should not be modified without consideration. 
 */
package com.eightylegs.customer.default80legs;

import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.net.URLDecoder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;



public class URLFunctions {
	public static final int MAX_URL_LENGTH = 800;
	public static final int MAX_OUTGOING_LINK_LENGTH = 500;
    public static final int MIN_URL_LENGTH = 10;
    
    
    //For seed URLs
	public static boolean isValidLink(String outLink){
        outLink = outLink.toLowerCase().trim();
        try {
        	URL aUrl = new URL(outLink);
        	if( outLink.equals("") 
	        	|| !outLink.contains("/")
	            || !outLink.startsWith("http")
	        	|| outLink.length() > MAX_URL_LENGTH
	            || outLink.length() < MIN_URL_LENGTH
	            || outLink.startsWith("http://localhost/")
	            || outLink.startsWith("https://localhost/")
	            || outLink.indexOf("\n") > 0
	            || outLink.indexOf("\r") > 0
	            || !isDomainValid(aUrl)
	           	)
	                return false;
	            else 
	                return true;
        }
        catch (MalformedURLException e) {
        	return false;
        }
    }
    
	//Stricter for outgoing links
	/*
	public static boolean isValidOutgoingLink(String outLink){
        outLink = outLink.toLowerCase().trim();
        try {
        	URL aUrl = new URL(outLink);
        	if( outLink.equals("") 
	        	|| outLink.contains("(") || outLink.contains(")") 
	        	|| outLink.contains("[") || outLink.contains("]")
	        	|| outLink.contains("<") || outLink.contains(">")
	        	|| !outLink.contains("/")
	            || !outLink.startsWith("http")
	        	|| outLink.length() > MAX_OUTGOING_LINK_LENGTH
	            || outLink.length() < MIN_URL_LENGTH
	            //|| hasBadCharacters(outLink)
	            || outLink.startsWith("http://localhost/")
	            || outLink.startsWith("https://localhost/")
	            || outLink.indexOf("\n") > 0
	            || outLink.indexOf("\r") > 0
	            || !isDomainValid(aUrl)
	           	)
	                return false;
	            else 
	                return true;
        }
        catch (MalformedURLException e) {
        	return false;
        }
    }
    */
	
	public static boolean isDomainValid ( String url ) {

		boolean isGood = true;
		URL urlObj = null;
		
		if (!url.startsWith("http"))
			return false;
		
		try {
			urlObj = new URL(url);
			isGood = isDomainValid(urlObj);
		}
		catch (MalformedURLException e) {
			return false;
		}
		
		
		return isGood;
	}
	
	public static boolean isDomainValid ( URL url ) {
		boolean isGood = true;
		String domain = url.getHost();
			
		//Check for length after the last period "."
		int tldLength = domain.length() - (domain.lastIndexOf(".") + 1);
		if (tldLength > 7 || tldLength < 1) { 
			return false;
		}
			
		if (!domain.contains(".") || domain.startsWith(".") || domain.contains(".."))
			return false;
			
		int firstPeriodPos = domain.indexOf(".");
		int curPos = 0;
		for ( char curChar : domain.toCharArray() ) {
			if (( curChar >= 'a' ) && ( curChar <= 'z' )) {
				// ok
			}
			else if (( curChar >= 'A' ) && ( curChar <= 'Z' )) {
				// ok
			}
			else if (( curChar >= '0' ) && ( curChar <= '9' )) {
				// ok
			}
			else if (( curChar == '.' ) || ( curChar == '-' )) {
				// ok
			}
			else if ((curChar == '_') && (curPos < firstPeriodPos)) {
				//OK
			}
			else {
				isGood = false;
				break;
			}
			curPos++;
		}//End of for loop
		
		return isGood;
	}
	
	
	public static boolean isValidLinkBeforeNormalization(String outLink) {
		String tempOutLink = outLink.trim().toLowerCase();
		
        if( tempOutLink.startsWith("javascript:") || tempOutLink.contains("mailto:") || tempOutLink.startsWith("about:") || tempOutLink.startsWith("ftp:") || tempOutLink.startsWith("#")
        		|| tempOutLink.matches("((?i)(.*?)\\s+\\+\\s*(.+?))|((?i)(.*?)\\s*\\+\\s+(.+?))") //Link with "+" and spaces before and after the sign: more likely it's a js
        		|| (getSubStringOccurances(tempOutLink, "http:") > 1)
        		|| (getSubStringOccurances(tempOutLink, "https:") > 1)
            ) 
        { 
            return false;
        }
        else {
            return true;
        }
	}
	
    public static Boolean isCrawlableLink(String link)	{
        Pattern pat = Pattern.compile("^.+\\.((png)|(jpg)|(jpeg)|(gif)|(bmp)|(tiff)|(ico)|(doc)|(xls)|(ppt)|(pdf)|(txt)|(dll)|(swf)|(mov)|(flv)|(mp3)|(mp4)|(wma)|(wmv)|(wav)|(asf)|(rm)|(ram)|(ra)|(avi)|(mpeg)|(mid)|(mpg)|(css)|(js))$");
        Matcher matcher = pat.matcher(link);
        return !matcher.matches();
    }
    
    public static String preProcessParsedLink(String outLink, URL originalUrl, URL redirectedUrl, boolean allowQueryStrings) throws Exception {
		outLink = URLFunctions.trimOutParsedLink(outLink);
		if (!outLink.equals("")) {
			if (isValidLinkBeforeNormalization(outLink)) {
				outLink = URLFunctions.normalizeParsedLink(outLink, originalUrl, redirectedUrl, allowQueryStrings);	
				if (!outLink.equals(originalUrl.toString()) && !outLink.equals(redirectedUrl.toString())) {
					return outLink;
				}
			}
		}
		return "";
	}
    
    /*
     * This function is called on both custom parseLinks() and default parseLinks(). It's also called for URLs in seed lists. 
     */
    
    //public static String postProcessParsedLink1(String outLink) throws Exception {
    public static String processRawLink(String outLink) throws Exception {
    	outLink = outLink.replaceAll("\n", "");
    	outLink = outLink.replaceAll("\r", "");
    	outLink = outLink.replaceAll("&amp;", "&");
    	outLink = outLink.replaceAll("\"", "%22");
    	outLink = outLink.replaceAll("'", "%27");
    	outLink = outLink.replaceAll("\\|", "%7C"); //Replace |
    	
    	
    	if (hasBadCharacters(outLink)) {
    		outLink = urlEncode(outLink);
    	}
    	
    	return outLink.trim();
    }
    
    /*	Return false if there is at least an ASCII > 128
     *	Not being used since 02/04/2010
     */
    public static boolean hasBadCharacters(String link) {
    	boolean found = false;
    	int i = 0;
    	while (i < link.length() && !found) {
    		char c = link.charAt( i );
		    int j = (int) c;
		    if (j >= 128) {
		    	found = true;
		    }
		    else {
		    	i++;
		    }
    	}
    	return found;
    }
    
    //URL originalUrl is not being used at the moment
    public static String normalizeParsedLink(String outLink, URL originalUrl, URL redirectedUrl, boolean allowQueryStrings) throws Exception {		
    	outLink = outLink.trim();
    	
    	if (!outLink.toLowerCase().startsWith("http")){
			if (outLink.startsWith("../")) {
				//outLink = redirectedUrl.getProtocol() + "://" + redirectedUrl.getHost() + outLink.replaceFirst("..", "");
				outLink = new URI(redirectedUrl.toString()).resolve(outLink).toString(); //Use URI.resolve() to resolve relative links
			}
			else if (outLink.startsWith("//")){ 
				//This is protocol relative URL. They point to an address, keeping the current protocol.
				//This means that when the base URI is http://a/b/c/d;p?q and you use //g, the relative reference is transformed to http://g.
				outLink = redirectedUrl.getProtocol() + ":" + outLink;
			}
			else if (outLink.startsWith("/")){
				outLink = redirectedUrl.getProtocol() + "://" + redirectedUrl.getHost() + outLink;
			}
			else {
				//Relative path, 5 scenarios:
				//(1): http://www.aaa.com   -- Link: a.html  --> http://www.aaa.com/a.html
				//(2): http://www.aaa.com/test.html -- Link: a.html --> http://www.aaa.com/a.html
				//(3): http://www.aaa.com/tricky/test.html -- Link: a.html --> http://www.aaa.com/tricky/a.html
				//(4): http://www.aaa.com/tricky -- Link: a.html --> http://www.aaa.com/a.html
				//(5): http://www.aaa.com/tricky/ -- Link: a.html --> http://www.aaa.com/tricky/a.html
				
				String redirectedUrlString = redirectedUrl.toString();
				//Remove query string for parent URL
				int firstQuestionMarkPos = redirectedUrlString.indexOf("?");
				if (firstQuestionMarkPos != -1)
					redirectedUrlString = redirectedUrlString.substring(0,firstQuestionMarkPos);
				
				char[] chArrayUrl = redirectedUrlString.toCharArray();
				int slashCount = 0;
				for (int i = 0; i < chArrayUrl.length; i++)
					if (chArrayUrl[i] == '/')
							slashCount++;
				
				if (redirectedUrlString.endsWith("/")) {
					outLink = redirectedUrlString + outLink; //(5)
				}
				else {
					if (slashCount <= 2) {	//(1)
						outLink = redirectedUrlString + '/' + outLink;
					}
					else {	// (2) (3) (4)
						String[] arrayUrl = redirectedUrlString.split("/");
						StringBuilder tempSourceUrlStringBuilder = new StringBuilder();
						int maxLength = arrayUrl.length-1;
							
						for (int i = 0; i < maxLength; i++)
							tempSourceUrlStringBuilder.append(arrayUrl[i].toString()).append("/");
						
						outLink = tempSourceUrlStringBuilder.append(outLink).toString();
					}
				}
			}
		}		
		
		if (allowQueryStrings == false) {
			int firstQuestionMarkPos = outLink.indexOf("?");
			if (firstQuestionMarkPos != -1)
				outLink = outLink.substring(0,firstQuestionMarkPos);
		}

		outLink = outLink.replaceAll("\t", " "); //Replace tab by space. Tab is used as delimiter. 
		
		if(outLink.contains("#")) {  //Remove anything after #
			outLink = outLink.substring(0, outLink.indexOf("#"));
		}
		
		outLink = outLink.replaceAll("\\\\", "/"); //Replace "\" with "/"
		outLink = outLink.replaceAll("/+", "/"); //Replace one or more "/" with "/"
		outLink = outLink.replaceFirst("/", "//");//For http:/ --> http://
		
		return outLink.trim();
	}
	
	
	//Remove ("href=" or "src=" or "<meta....") and quotes
	public static String trimOutParsedLink(String link)
	{	
		link = link.trim();
		if (link.toLowerCase().startsWith("href"))
			link = link.substring(4, link.length());
		else if (link.toLowerCase().startsWith("src"))
			link = link.substring(3, link.length());
		else if (link.toLowerCase().startsWith("<meta")) {  //Example: <meta http-equiv="Refresh" content="0; url=http://www.example.com/">
			int startPos = link.toLowerCase().indexOf("url");
			int endPos = link.toLowerCase().indexOf(">", startPos);
			link = link.substring(startPos + "url".length(), endPos);
		}
		
		link = link.trim();
		while (link.endsWith(">")) {
			link = link.substring(0, link.length()-1);
			link = link.trim();
		}
		
		if (link.toLowerCase().startsWith("="))
			link = link.substring(1, link.length());
		
		link = link.trim();
		while (link.startsWith("\"") || link.startsWith("'")) {
			link = link.substring(1, link.length());
			link = link.trim();
		}
		
		while (link.endsWith("\"") || link.endsWith("'")) {
			link = link.substring(0, link.length()-1);
			link = link.trim();
		}
		return link.trim();
	}
	
	public static boolean hasSameDomainAsParent(URL parentURL, URL childURL, boolean fullyQualified, boolean restrictedHost) throws Exception {
		boolean retVal = false;
		
		if (fullyQualified == false) {
			restrictedHost = false;
		}
		
		String parentDomain = parentURL.getHost().toLowerCase();
		String childDomain = childURL.getHost().toLowerCase();

		//Remove "www." if restrictedHost == false
		if ( parentDomain.startsWith("www.") && restrictedHost == false )
			parentDomain = parentDomain.substring(4, parentDomain.length());
		
		if ( childDomain.startsWith("www.") && restrictedHost == false )
			childDomain = childDomain.substring(4, childDomain.length());
		
		if (fullyQualified == true) {
			if (childDomain.equals(parentDomain)) {
				retVal = true;
			}
			else {
				retVal = false;
			}
		}
		else {
			String[] parentDomainTokens = parentDomain.split("\\.");
			String[] reverseParentDomainTokens = reverseStringArray(parentDomainTokens);
			String[] childDomainTokens = childDomain.split("\\.");
			String[] reverseChildDomainTokens = reverseStringArray(childDomainTokens);
			
			int i = 0;
			retVal = true;
			while ( i < reverseChildDomainTokens.length && i < reverseParentDomainTokens.length ) {
				if (!reverseChildDomainTokens[i].equals(reverseParentDomainTokens[i])) {
					retVal = false;
					break;
				}
				i++;
			}
		}
		return retVal;
	}
	
	public static boolean matchesPageURL ( String urlString, Pattern customerCrawlRegexPattern ) {
		if ( customerCrawlRegexPattern == null )
			return true;
		
		Matcher curMatcher = customerCrawlRegexPattern.matcher(urlString);
		if ( curMatcher.find() == true )
			return true;
		return false;
	}
		
	/*
	public static boolean needUrlEncode1(String url) throws Exception {
		if (url.indexOf(" ") != -1) { //If this URL has space
			return true;
		}
		
		int curPos = url.indexOf("%");
		while (curPos < url.length() && curPos != -1) { //If this URL has %, check to see if the next two characters are valid hex
			if (curPos + 3 <= url.length()) {
				String temp = url.substring(curPos, curPos + 3);
				temp = temp.toLowerCase();
				//System.out.println("Checking=" + temp);
				boolean isValidHex = false;
				for (int i = 0; i < hex.length; i++) {
					if (hex[i].equals(temp)) {
						isValidHex = true;
						break;
					}
				}
				
				if (isValidHex == false) {
					return true;
				}
			}
			
			curPos = url.indexOf("%", curPos + 1);
		}
			
		return false;
	}
	*/
	
	
	/*
	 * This function expects the "file" part of a URL
	 */
	public static String urlDecode(String encodedString) {
		try {
			return URLDecoder.decode(encodedString, "UTF-8");
		}
		catch (Exception e) {
			return encodedString;
		}
	}
	
	public static String urlEncode( String urlString ) throws Exception {
		URL url = new URL(urlString);
		String urlFile = url.getFile();
		urlFile = urlEncodeByCharacter(urlFile);
		return new URL(url.getProtocol(), url.getHost(), urlFile).toString();
	}
	
	private static int getSubStringOccurances(String string, String subString ) {
		  int count = 0 ;
		  int pos = -1 ;
		  while ((pos = string.indexOf(subString, pos + 1)) > -1) { 
			  count++ ;
		  }

		  return count;
	}
	
	private static String[] reverseStringArray(String[] array) {
		String[] ret = new String[array.length];
		int i = 0;
		while (i < array.length) {
			ret[i] = array[array.length - i - 1];
			i++;
		}
		return ret;
	}
	
	/*
	 * This function URL-encodes a string in character by character fashion. It is best to break a URL into components and only pass the path, query, and anchor (optional) into the function for encoding
	 * because if entire URL is passed in, the "http://" will also be encoded. 
	 */
	private static String urlEncodeByCharacter(String s) {
		if (s == null) {
			return null;
		}
		
	    StringBuffer sbuf = new StringBuffer();
	    int len = s.length();
	    for (int i = 0; i < len; i++) {
	        int ch = s.charAt(i);
	        if ('A' <= ch && ch <= 'Z') { // 'A'..'Z'
	            sbuf.append((char)ch);
	        } else if ('a' <= ch && ch <= 'z') { // 'a'..'z'
	            sbuf.append((char)ch);
	        } else if ('0' <= ch && ch <= '9') { // '0'..'9'
	            sbuf.append((char)ch);
	        } else if (ch == ' ') { // space
	            sbuf.append('+');
	        } else if (ch == '-' || ch == '_' // unreserved
	                || ch == '.' || ch == '!'
	                || ch == '~' || ch == '*'
	                || ch == '\\' || ch == '('
	                || ch == ')' || ch == '/' 
		        || ch == '&' || ch == '='
		        || ch == '?' || ch == '+') {
	            sbuf.append((char)ch);
	        } else if (ch <= 0x007f) { // other ASCII
	            sbuf.append(hex[ch]);
	        } else if (ch <= 0x07FF) { // non-ASCII <= 0x7FF
	            sbuf.append(hex[0xc0 | (ch >> 6)]);
	            sbuf.append(hex[0x80 | (ch & 0x3F)]);
	        } else { // 0x7FF < ch <= 0xFFFF
	            sbuf.append(hex[0xe0 | (ch >> 12)]);
	            sbuf.append(hex[0x80 | ((ch >> 6) & 0x3F)]);
	            sbuf.append(hex[0x80 | (ch & 0x3F)]);
	        }
	    }
	    return sbuf.toString();
	}
	 
	 
	// Hex constants.
	final static String[] hex = {
	    "%00", "%01", "%02", "%03", "%04", "%05", "%06", "%07",
	    "%08", "%09", "%0a", "%0b", "%0c", "%0d", "%0e", "%0f",
	    "%10", "%11", "%12", "%13", "%14", "%15", "%16", "%17",
	    "%18", "%19", "%1a", "%1b", "%1c", "%1d", "%1e", "%1f",
	    "%20", "%21", "%22", "%23", "%24", "%25", "%26", "%27",
	    "%28", "%29", "%2a", "%2b", "%2c", "%2d", "%2e", "%2f",
	    "%30", "%31", "%32", "%33", "%34", "%35", "%36", "%37",
	    "%38", "%39", "%3a", "%3b", "%3c", "%3d", "%3e", "%3f",
	    "%40", "%41", "%42", "%43", "%44", "%45", "%46", "%47",
	    "%48", "%49", "%4a", "%4b", "%4c", "%4d", "%4e", "%4f",
	    "%50", "%51", "%52", "%53", "%54", "%55", "%56", "%57",
	    "%58", "%59", "%5a", "%5b", "%5c", "%5d", "%5e", "%5f",
	    "%60", "%61", "%62", "%63", "%64", "%65", "%66", "%67",
	    "%68", "%69", "%6a", "%6b", "%6c", "%6d", "%6e", "%6f",
	    "%70", "%71", "%72", "%73", "%74", "%75", "%76", "%77",
	    "%78", "%79", "%7a", "%7b", "%7c", "%7d", "%7e", "%7f",
	    "%80", "%81", "%82", "%83", "%84", "%85", "%86", "%87",
	    "%88", "%89", "%8a", "%8b", "%8c", "%8d", "%8e", "%8f",
	    "%90", "%91", "%92", "%93", "%94", "%95", "%96", "%97",
	    "%98", "%99", "%9a", "%9b", "%9c", "%9d", "%9e", "%9f",
	    "%a0", "%a1", "%a2", "%a3", "%a4", "%a5", "%a6", "%a7",
	    "%a8", "%a9", "%aa", "%ab", "%ac", "%ad", "%ae", "%af",
	    "%b0", "%b1", "%b2", "%b3", "%b4", "%b5", "%b6", "%b7",
	    "%b8", "%b9", "%ba", "%bb", "%bc", "%bd", "%be", "%bf",
	    "%c0", "%c1", "%c2", "%c3", "%c4", "%c5", "%c6", "%c7",
	    "%c8", "%c9", "%ca", "%cb", "%cc", "%cd", "%ce", "%cf",
	    "%d0", "%d1", "%d2", "%d3", "%d4", "%d5", "%d6", "%d7",
	    "%d8", "%d9", "%da", "%db", "%dc", "%dd", "%de", "%df",
	    "%e0", "%e1", "%e2", "%e3", "%e4", "%e5", "%e6", "%e7",
	    "%e8", "%e9", "%ea", "%eb", "%ec", "%ed", "%ee", "%ef",
	    "%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7",
	    "%f8", "%f9", "%fa", "%fb", "%fc", "%fd", "%fe", "%ff"
	};
}
